<!DOCTYPE html>
<html xmlns="http://www.w3.org/1999/xhtml">
  <head>

    <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
    <meta content="Cask Data, Inc." name="author" />
<meta content="Copyright © 2015-2017 Cask Data, Inc." name="copyright" />


    <meta name="git_release" content="6.1.1">
    <meta name="git_hash" content="05fbac36f9f7aadeb44f5728cea35136dbc243e5">
    <meta name="git_timestamp" content="2020-02-09 08:22:47 +0800">
    <title>Transaction Service Maintenance</title>

    <link rel="stylesheet" href="../_static/cdap-bootstrap.css" type="text/css" />
    <link rel="stylesheet" href="../_static/pygments.css" type="text/css" />
    <link rel="stylesheet" href="../_static/bootstrap-3.3.6/css/bootstrap.min.css" type="text/css" />
    <link rel="stylesheet" href="../_static/bootstrap-3.3.6/css/bootstrap-theme.min.css" type="text/css" />
    <link rel="stylesheet" href="../_static/css/bootstrap-sphinx.css" type="text/css" />
    <link rel="stylesheet" href="../_static/css/cdap-dynamicscrollspy-4.css" type="text/css" />
    <link rel="stylesheet" href="../_static/css/jquery.mCustomScrollbar.css" type="text/css" />
    <link rel="stylesheet" href="../_static/css/cdap-jquery.mCustomScrollbar.css" type="text/css" />
    <link rel="stylesheet" href="../_static/css/abixTreeList-2.css" type="text/css" />
    <link rel="stylesheet" href="../_static/cdap-bootstrap.css" type="text/css" />

    <script type="text/javascript">
      var DOCUMENTATION_OPTIONS = {
        URL_ROOT:    '',
        VERSION:     '6.1.1',
        COLLAPSE_INDEX: false,
        FILE_SUFFIX: '.html',
        HAS_SOURCE:  false
      };
    </script>
    <script type="text/javascript" src="../_static/jquery.js"></script>
    <script type="text/javascript" src="../_static/underscore.js"></script>
    <script type="text/javascript" src="../_static/doctools.js"></script>
    <script type="text/javascript" src="../_static/language_data.js"></script>

    <link rel="shortcut icon" href="../_static/favicon.ico"/>
    <link rel="index" title="Index" href="../genindex.html" />
    <link rel="search" title="Search" href="../search.html" />
    <link rel="top" title="Cask Data Application Platform 6.1.1 Documentation" href="../index.html" />
    <link rel="up" title="Operations" href="index.html" />
    <link rel="next" title="CDAP UI" href="cdap-ui.html" />
    <link rel="prev" title="Resource Guarantees for CDAP Programs in YARN" href="resource-guarantees.html" />
    <!-- block extrahead -->
    <meta charset='utf-8'>
    <meta http-equiv='X-UA-Compatible' content='IE=edge,chrome=1'>
    <meta name='viewport' content='width=device-width, initial-scale=1.0, maximum-scale=1'>
    <meta name="apple-mobile-web-app-capable" content="yes">
    <!-- block extrahead end -->

</head>
<body role="document">

<!-- block navbar -->
<div id="navbar" class="navbar navbar-inverse navbar-default navbar-fixed-top">
    <div class="container-fluid">
      <div class="row">
        <div class="navbar-header">
          <!-- .btn-navbar is used as the toggle for collapsed navbar content -->
          <a class="navbar-brand" href="../table-of-contents/../../index.html">
            <span><img alt="CDAP logo" src="../_static/cdap_logo.svg"/></span>
          </a>

          <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".nav-collapse">
            <span class="icon-bar"></span>
            <span class="icon-bar"></span>
            <span class="icon-bar"></span>
          </button>

          <div class="pull-right">
            <div class="dropdown version-dropdown">
              <a href="#" class="dropdown-toggle" data-toggle="dropdown"
                role="button" aria-haspopup="true" aria-expanded="false">
                v 6.1.1 <span class="caret"></span>
              </a>
              <ul class="dropdown-menu">
                <li><a href="//docs.cdap.io/cdap/5.1.2/en/index.html">v 5.1.2</a></li>
                <li><a href="//docs.cdap.io/cdap/4.3.4/en/index.html">v 4.3.4</a></li>
              </ul>
            </div>
          </div>
          <form class="navbar-form navbar-right navbar-search" action="../search.html" method="get">
            <div class="form-group">
              <div class="navbar-search-image material-icons"></div>
              <input type="text" name="q" class="form-control" placeholder="  Search" />
            </div>
            <input type="hidden" name="check_keywords" value="yes" />
            <input type="hidden" name="area" value="default" />
          </form>

          <div class="collapse navbar-collapse nav-collapse navbar-right navbar-navigation">
            <ul class="nav navbar-nav"><li class="docsite-nav-tab-container"><a class="docsite-nav-tab-link " href="../table-of-contents/../../index.html">简介</a></li><li class="docsite-nav-tab-container"><a class="docsite-nav-tab-link current" href="../table-of-contents/../../guides.html">手册</a></li><li class="docsite-nav-tab-container"><a class="docsite-nav-tab-link " href="../table-of-contents/../../reference-manual/index.html">参考</a></li><li class="docsite-nav-tab-container"><a class="docsite-nav-tab-link " href="../table-of-contents/../../faqs/index.html">帮助</a></li>
            </ul>
          </div>

        </div>
      </div>
    </div>
  </div><!-- block navbar end -->
<!-- block main content -->
<div class="main-container container">
  <div class="row"><div class="col-md-2">
      <div id="sidebar" class="bs-sidenav scrollable-y-outside" role="complementary">
<!-- theme_manual: admin-manual -->
<!-- theme_manual_highlight: guides -->
<!-- sidebar_title_link: ../table-of-contents/../../guides.html -->

  <div role="note" aria-label="manuals links"><h3><a href="../table-of-contents/../../guides.html">Guides</a></h3>

    <ul class="this-page-menu">
      <li class="toctree-l1"><a href="../table-of-contents/../../user-guide/index.html" rel="nofollow">用户手册</a>
      </li>
      <li class="toctree-l1"><a href="../table-of-contents/../../developer-manual/index.html" rel="nofollow">开发手册</a>
      </li>
      <li class="toctree-l1"><b><a href="../table-of-contents/../../admin-manual/index.html" rel="nofollow">管理手册</a></b>
      <nav class="pagenav">
      <ul class="current">
<li class="toctree-l1"><a class="reference internal" href="../index.html"> Introduction</a></li>
<li class="toctree-l1"><a class="reference internal" href="../cdap-components.html"> CDAP Components</a></li>
<li class="toctree-l1"><a class="reference internal" href="../deployment-architectures.html"> Deployment Architectures</a></li>
<li class="toctree-l1"><a class="reference internal" href="../hadoop-compatibility.html"> Hadoop Compatibility</a></li>
<li class="toctree-l1"><a class="reference internal" href="../cdap-hadoop-compatibility.html"> CDAP and Hadoop Compatibility</a></li>
<li class="toctree-l1"><a class="reference internal" href="../system-requirements.html"> System Requirements</a></li>
<li class="toctree-l1"><a class="reference internal" href="../installation/index.html"> Installation</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../installation/cloudera.html">Cloudera Manager</a></li>
<li class="toctree-l2"><a class="reference internal" href="../installation/emr.html">Amazon EMR</a></li>
<li class="toctree-l2"><a class="reference internal" href="../installation/ambari.html">Apache Ambari</a></li>
<li class="toctree-l2"><a class="reference internal" href="../installation/mapr.html">MapR</a></li>
<li class="toctree-l2"><a class="reference internal" href="../installation/azure-hdinsight.html">Microsoft Azure HDInsight</a></li>
<li class="toctree-l2"><a class="reference internal" href="../installation/packages.html">Packages</a></li>
<li class="toctree-l2"><a class="reference internal" href="../installation/replication.html">Replication</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../incompatibilities.html"> Incompatibilities</a></li>
<li class="toctree-l1"><a class="reference internal" href="../upgrading/index.html"> Upgrading</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../upgrading/cloudera.html">Cloudera Manager</a></li>
<li class="toctree-l2"><a class="reference internal" href="../upgrading/ambari.html">Apache Ambari</a></li>
<li class="toctree-l2"><a class="reference internal" href="../upgrading/mapr.html">MapR</a></li>
<li class="toctree-l2"><a class="reference internal" href="../upgrading/packages.html">Packages</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../security/index.html"> Security</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../security/perimeter-security.html">Perimeter Security</a></li>
<li class="toctree-l2"><a class="reference internal" href="../security/authorization.html">Authorization</a></li>
<li class="toctree-l2"><a class="reference internal" href="../security/impersonation.html">Impersonation</a></li>
<li class="toctree-l2"><a class="reference internal" href="../security/system-services.html">Enabling SSL for System Services</a></li>
<li class="toctree-l2"><a class="reference internal" href="../security/secure-storage.html">Secure Storage</a></li>
</ul>
</li>
<li class="toctree-l1 current"><a class="reference internal" href="index.html"> Operations</a><ul class="current">
<li class="toctree-l2"><a class="reference internal" href="logging.html"> Logging and Monitoring</a></li>
<li class="toctree-l2"><a class="reference internal" href="metrics.html"> Metrics</a></li>
<li class="toctree-l2"><a class="reference internal" href="operations-dashboard.html"> Dashboard and Reports</a></li>
<li class="toctree-l2"><a class="reference internal" href="preferences.html"> Preferences and Runtime Arguments</a></li>
<li class="toctree-l2"><a class="reference internal" href="scaling-instances.html"> Scaling Instances</a></li>
<li class="toctree-l2"><a class="reference internal" href="resource-guarantees.html"> Resource Guarantees in YARN</a></li>
<li class="toctree-l2 current"><a class="current reference internal" href="#"> Transaction Service Maintenance</a></li>
<li class="toctree-l2"><a class="reference internal" href="cdap-ui.html"> CDAP UI</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../appendices/index.html"> Appendices</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../appendices/cdap-site.html"> Appendix: cdap-site.xml</a></li>
<li class="toctree-l2"><a class="reference internal" href="../appendices/cdap-security.html"> Appendix: cdap-security.xml</a></li>
<li class="toctree-l2"><a class="reference internal" href="../appendices/minimal-cdap-site.html"> Appendix: Minimal cdap-site.xml</a></li>
<li class="toctree-l2"><a class="reference internal" href="../appendices/hbase-ddl-executor.html"> Appendix: HBaseDDLExecutor</a></li>
</ul>
</li>
</ul>
</nav>
      </li>
      <li class="toctree-l1"><a href="../table-of-contents/../../integrations/index.html" rel="nofollow">集成手册</a>
      </li>
      <li class="toctree-l1"><a href="../table-of-contents/../../examples-manual/index.html" rel="nofollow">最佳实践</a>
      </li>
    </ul>
  </div></div>
    </div><div class="col-md-8 content" id="main-content">
    
  <div class="section" id="transaction-service-maintenance">
<span id="tx-maintenance"></span><h1>Transaction Service Maintenance<a class="headerlink" href="#transaction-service-maintenance" title="Permalink to this headline">🔗</a></h1>
<div class="section" id="pruning-invalid-transactions">
<span id="tx-maintenance-pruning-invalid-transactions"></span><h2>Pruning Invalid Transactions<a class="headerlink" href="#pruning-invalid-transactions" title="Permalink to this headline">🔗</a></h2>
<p>The Transaction Service keeps track of all invalid transactions so as to exclude their
writes from all future reads. Over time, this <em>invalid</em> list can grow and lead to
performance degradation. To avoid that, you can prune the <em>invalid</em> list after the data of
invalid transactions has been removed—the removal of which happens during major
HBase compactions of the transactional tables.</p>
<p>To prune the invalid list manually, follow these steps:</p>
<ol class="arabic">
<li><p class="first">Find the minimum transaction state cache reload time across all HBase region servers,
by finding the last occurrence of the following line in the HBase region server logs:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="go">[&lt;instance.name&gt;] Transaction state reloaded with snapshot</span>
</pre></div>
</div>
<p><code class="docutils literal notranslate"><span class="pre">&lt;instance.name&gt;</span></code> is the unique identifier for the CDAP instance being pruned as defined in
<a class="reference internal" href="../appendices/cdap-site.html#appendix-cdap-site-xml"><span class="std std-ref">cdap-site.xml</span></a>. The default value for it is <code class="docutils literal notranslate"><span class="pre">cdap</span></code>.</p>
<p>Run the following command on each region server to get the transaction cache state reload time
after replacing the <code class="docutils literal notranslate"><span class="pre">&lt;instance.name&gt;</span></code> with the right value:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="go">grep -F &quot;[&lt;instance.name&gt;] Transaction state reloaded with snapshot&quot; &lt;region-server-log-file&gt; | tail -1</span>
</pre></div>
</div>
<p>This should give lines as shown below. Each line below represents one line from each
region server:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="go">15/08/22 00:22:34 INFO coprocessor.TransactionStateCache: [cdap] Transaction state reloaded with snapshot from 1440202895873</span>
<span class="go">15/08/22 00:22:42 INFO coprocessor.TransactionStateCache: [cdap] Transaction state reloaded with snapshot from 1440202956306</span>
<span class="go">15/08/22 00:22:44 INFO coprocessor.TransactionStateCache: [cdap] Transaction state reloaded with snapshot from 1440202956306</span>
<span class="go">15/08/22 00:22:47 INFO coprocessor.TransactionStateCache: [cdap] Transaction state reloaded with snapshot from 1440202956306</span>
<span class="go">15/08/22 00:23:34 INFO coprocessor.TransactionStateCache: [cdap] Transaction state reloaded with snapshot from 1440202956306</span>
</pre></div>
</div>
<p>Pick the minimum time across all region servers. In this case, <code class="docutils literal notranslate"><span class="pre">1440202895873</span></code>.</p>
</li>
<li><p class="first">Run a flush and a major compaction on all CDAP transactional tables.</p>
</li>
<li><p class="first">Wait for the major compaction to complete.</p>
</li>
<li><p class="first">Get the minimum time to obtain the <code class="docutils literal notranslate"><span class="pre">pruneTime</span></code>. In this case, <code class="docutils literal notranslate"><span class="pre">1440202895873</span></code>.</p>
</li>
<li><p class="first">If the CDAP tables are replicated to other clusters, see the section below
(<a class="reference internal" href="#tx-maintenance-pruning-replicated"><span class="std std-ref">Pruning Invalid Transactions in a Replicated Cluster</span></a>) to obtain the <code class="docutils literal notranslate"><span class="pre">pruneTime</span></code> for the slave
clusters.</p>
</li>
<li><p class="first">The final <code class="docutils literal notranslate"><span class="pre">pruneTime</span></code> is the minimum <code class="docutils literal notranslate"><span class="pre">pruneTime</span></code> across all replicated clusters (if
any); let this be time <code class="docutils literal notranslate"><span class="pre">t</span></code>.</p>
</li>
</ol>
<p>Now, the invalid transaction list can be safely pruned until <code class="docutils literal notranslate"><span class="pre">(t</span> <span class="pre">-</span> <span class="pre">1</span> <span class="pre">day)</span></code> using a call
to <span class="xref std std-ref">truncate invalid transactions before a specific time</span>.</p>
<p>The current length of the invalid transaction list can be obtained using a call to
<span class="xref std std-ref">retrieve the number of invalid transactions</span>.</p>
<div class="section" id="pruning-invalid-transactions-in-a-replicated-cluster">
<span id="tx-maintenance-pruning-replicated"></span><h3>Pruning Invalid Transactions in a Replicated Cluster<a class="headerlink" href="#pruning-invalid-transactions-in-a-replicated-cluster" title="Permalink to this headline">🔗</a></h3>
<ol class="arabic simple">
<li>Copy over the latest transaction snapshots from the master cluster to the slave cluster.</li>
<li>Wait for three to four minutes for the latest transaction state to be reloaded from the
snapshot.</li>
<li>Run steps 1 to 5 from the above section (<a class="reference internal" href="#tx-maintenance-pruning-invalid-transactions"><span class="std std-ref">Pruning Invalid Transactions</span></a>)
on the slave cluster to find the <code class="docutils literal notranslate"><span class="pre">pruneTime</span></code> for that slave cluster.</li>
</ol>
</div>
<div class="section" id="automated-pruning-of-invalid-transactions">
<h3>Automated Pruning of Invalid Transactions<a class="headerlink" href="#automated-pruning-of-invalid-transactions" title="Permalink to this headline">🔗</a></h3>
<p>From CDAP 4.1 onwards, CDAP supports automated pruning of the invalid transactions list.
It is turned off by default. Please contact <a class="reference external" href="mailto:support&#37;&#52;&#48;cask&#46;co">support<span>&#64;</span>cask<span>&#46;</span>co</a> if you are
interested in turning it on.</p>
<p>For automated pruning to work in a secure Hadoop cluster with authorization enabled,
CDAP needs to be able to list all CDAP tables and the table descriptors of all CDAP tables in HBase.
If CDAP is not able to list the table descriptors of all CDAP tables, running automated pruning
can lead to data inconsistency.</p>
<p>Note that the automated pruning in 4.1 works only on a <em>non-replicated</em> cluster.</p>
</div>
</div>
</div>

</div>
    <div class="col-md-2">
      <div id="right-sidebar" class="bs-sidenav scrollable-y" role="complementary">
        <div id="localtoc-scrollspy">
        </div>
      </div>
    </div></div>
</div>
<!-- block main content end -->
<!-- block footer -->
<footer class="footer">
      <div class="container">
        <div class="row">
          <div class="col-md-2 footer-left"><a title="Resource Guarantees for CDAP Programs in YARN" href="resource-guarantees.html" />Previous</a></div>
          <div class="col-md-8 footer-center"><a class="footer-tab-link" href="../table-of-contents/../../reference-manual/licenses/index.html">Copyright</a> &copy; 2014-2020 Cask Data, Inc.&bull; <a class="footer-tab-link" href="//docs.cask.co/cdap/6.1.1/cdap-docs-6.1.1-web.zip" rel="nofollow">Download</a> an archive or
<a class="footer-tab-link" href="//docs.cask.co/cdap">switch the version</a> of the documentation
          </div>
          <div class="col-md-2 footer-right"><a title="CDAP UI" href="cdap-ui.html" />Next</a></div>
        </div>
      </div>
    </footer>
<!-- block footer end -->
<script type="text/javascript" src="../_static/bootstrap-3.3.6/js/bootstrap.min.js"></script><script type="text/javascript" src="../_static/js/bootstrap-sphinx.js"></script><script type="text/javascript" src="../_static/js/abixTreeList-2.js"></script><script type="text/javascript" src="../_static/js/cdap-dynamicscrollspy-4.js"></script><script type="text/javascript" src="../_static/js/cdap-version-menu.js"></script><script type="text/javascript" src="../_static/js/copy-to-clipboard.js"></script><script type="text/javascript" src="../_static/js/jquery.mousewheel.min.js"></script><script type="text/javascript" src="../_static/js/jquery.mCustomScrollbar.js"></script><script type="text/javascript" src="../_static/js/js.cookie.js"></script><script type="text/javascript" src="../_static/js/tabbed-parsed-literal-0.2.js"></script><script type="text/javascript" src="../_static/js/cdap-onload-javascript.js"></script><script type="text/javascript" src="../_static/js/cdap-version-menu.js"></script>
    <script src="https://cdap.gitee.io/docs/cdap/json-versions.js"/></script>
  </body>
</html>